package wikifier;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.json.JSONArray;
import org.json.JSONException;

import org.w3c.dom.*;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import wikifier.formatters.AbstractFormatter;
import wikifier.formatters.DefinitionFormatter;
import wikifier.formatters.LinkFormatter;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.*;
import java.net.*;

import java.util.logging.Logger;

import com.google.wave.api.*;

/*Copyright (C) 2009  Joao Neves

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.*/


@SuppressWarnings("serial")
public class WikifierServlet extends AbstractRobotServlet {

	
	private static final Logger log = Logger.getLogger(WikifierServlet.class.getName());

	//Wikipedia search URL
	private static String WIKISEARCH = "http://%s.wikipedia.org/w/api.php?" +
		"action=opensearch&search=%s&format=xml&limit=100";
	
	// Regexs for catching the syntax
	private static Pattern WIKIFY_EXPR = Pattern.compile("(?<=<wikify(-([\\w]{2}))? )([\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+)>");
	private static Pattern WIKIDEF_EXPR = Pattern.compile("(?<=<wikidef(-([\\w]{2}))? )([\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+)>");
	private static Pattern WIKIFY_ALL = Pattern.compile("<wikify(-([\\w]{2}))? [\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+>");
	private static Pattern WIKIDEF_ALL = Pattern.compile("<wikidef(-([\\w]{2}))? [\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+>");
	
	//Experimental, DO NOT USE
	private static Pattern EXPERIMENTAL_EXPR = Pattern.compile("(?<=<wikiexp(-([\\w]{2}))? )([\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+)>");
	private static Pattern EXPERIMENTAL_ALL = Pattern.compile("<wikiexp(-([\\w]{2}))? [\\(\\)\\p{Z}\\p{L}\\p{N}\\p{M}\\p{P}]+>");
	
	//EXPERIMENTAL
	private static String GADGET_URL = "http://wikifier.appspot.com/assets/pickergadget.xml";
	
	private static enum Types { WIKIFY, WIKIDEF };
	
	/**
	 * Returns URL or description according to the type of query
	 * @param node The XML node containing the information
	 * @param repltype The type of replacement (URL or definition)
	 * @return URL or description according to the type of query
	 */
	private Node getData(org.w3c.dom.Element node, Types repltype){
		Node datanode;
		switch(repltype){
		case WIKIFY:
			datanode = node.getElementsByTagName("Url").item(0);
			//return URLDecoder.decode(datanode.getChildNodes().item(0).getNodeValue(), "UTF-8");
			break;
		case WIKIDEF:
			datanode = node.getElementsByTagName("Description").item(0);
			//return new String(datanode.getChildNodes().item(0).getNodeValue().getBytes("UTF-8"), "UTF-8");
			break;
		default:
			return null;
		}
		return datanode;
	}
	
	/**
	 * Parses XML data returned from a query to the Wikipedia API
	 * @param data The XML query result
	 * @param query The original query
	 * @param repltype Type of replacement (to URL or definition)
	 * @return String with the text to be used on replacement
	 * @throws SAXException
	 * @throws IOException
	 * @throws ParserConfigurationException
	 */
	private String parseData(String data, String query, Types repltype) throws SAXException, IOException, ParserConfigurationException{

		query = query.replaceAll("%20", " ");

		DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
		DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
		InputSource source = new InputSource();
		source.setCharacterStream(new StringReader(data));
		Document doc = docBuilder.parse(source);

		NodeList items = doc.getElementsByTagName("Item");
		Node first = null;

		for(int i = 0; i < items.getLength(); i++){
			Node item = items.item(i);
			if(item.getNodeType() == Node.ELEMENT_NODE){
				if(first == null)
					first = item;
				org.w3c.dom.Element node = (org.w3c.dom.Element)item;
				org.w3c.dom.Element text = (org.w3c.dom.Element)(node.getElementsByTagName("Text").item(0));
				if(text.getChildNodes().item(0).getNodeValue().equalsIgnoreCase(query) || items.getLength() == 1){
					Node datanode = getData(node, repltype);
					return URLDecoder.decode(datanode.getChildNodes().item(0).getNodeValue(), "UTF-8");
				}
			}
		}
		if(items.getLength() > 0) {
			// Return the first non-matching result
			org.w3c.dom.Element node = (org.w3c.dom.Element)first;
			Node datanode = getData(node, repltype);
			return URLDecoder.decode(datanode.getChildNodes().item(0).getNodeValue(), "UTF-8");
		}
		return null;
	}
	
	/**
	 * Replaces all marked up text in the blip with a URL or definition
	 * @param blip The submitted blip
	 * @param repltype The kind of replacement to do
	 * @throws SAXException
	 * @throws ParserConfigurationException
	 * @throws UnsupportedEncodingException
	 */
	private void replaceText(Blip blip, Types repltype) throws SAXException, ParserConfigurationException, UnsupportedEncodingException{

		while(true){
			String text = blip.getDocument().getText();
			Matcher textmatcher = null;
			Matcher allmatcher = null;
			switch(repltype){
			case WIKIFY:
				textmatcher = WIKIFY_EXPR.matcher(text);
				allmatcher = WIKIFY_ALL.matcher(text);
				break;
			case WIKIDEF:
				textmatcher = WIKIDEF_EXPR.matcher(text);
				allmatcher = WIKIDEF_ALL.matcher(text);
				break;
			default:
				return;
			}

			if(textmatcher != null && allmatcher != null && textmatcher.find() && allmatcher.find()){
				String topic = new String(textmatcher.group(3).getBytes("UTF-8"), "UTF-8");
				String lang = textmatcher.group(2);
				if(lang == null)
					lang = "en";
				topic = topic.replaceAll(" ", "%20");
				URL searchurl;
				try {
					searchurl = new URL(String.format(WIKISEARCH, lang, topic));
					BufferedReader reader = new BufferedReader(new InputStreamReader(searchurl.openStream(), "UTF-8"));
					String data = "";
					String line = "";
					while((line = reader.readLine()) != null){
						data += line;
					}
					reader.close();
					
					String repl = parseData(data, textmatcher.group(3), repltype);
					AbstractFormatter formatter = null;
					
					switch(repltype){
					case WIKIFY:
						formatter = new LinkFormatter();
						break;
					case WIKIDEF:
						formatter = new DefinitionFormatter();
						break;
					default:
						break;
					}

					if(repl != null){
						String newtext = formatter.format(textmatcher.group(3), repl);
						Range range = new Range(allmatcher.start(), allmatcher.end());
						TextView textview = blip.getDocument();
						textview.replace(range, newtext);
					} else {
						String newtext = String.format("%s [no Wikipedia article]", textmatcher.group(3));
						Range range = new Range(allmatcher.start(), allmatcher.end());
						TextView textview = blip.getDocument();
						textview.replace(range, newtext);
					}

				} catch (MalformedURLException e) {
					log.severe(e.getMessage());
				} catch (IOException e) {
					log.severe(e.getMessage());
				}

			} else {
				return;
			}
		}
	}
	
	/**
	 * Returns all topics resulting from the given search
	 * @param data XML String containing the results
	 * @param query Original query
	 * @return All topics relevant to the query, or null if none found
	 * @throws DOMException
	 * @throws SAXException
	 * @throws IOException
	 * @throws ParserConfigurationException
	 */
	private ArrayList<ArrayList<String>> getAllTopics(String data, String query) throws DOMException, SAXException, IOException, ParserConfigurationException{
		query = query.replaceAll("%20", " ");

		DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
		DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
		InputSource source = new InputSource();
		source.setCharacterStream(new StringReader(data));
		Document doc = docBuilder.parse(source);
		
		ArrayList<ArrayList<String>> topics = new ArrayList<ArrayList<String>>();
		ArrayList<String> names = new ArrayList<String>();
		ArrayList<String> info = new ArrayList<String>();

		NodeList items = doc.getElementsByTagName("Item");

		for(int i = 0; i < items.getLength(); i++){
			Node item = items.item(i);
			if(item.getNodeType() == Node.ELEMENT_NODE){
				org.w3c.dom.Element node = (org.w3c.dom.Element)item;
				org.w3c.dom.Element text = (org.w3c.dom.Element)(node.getElementsByTagName("Text").item(0));
				org.w3c.dom.Element urlnode = (org.w3c.dom.Element)(node.getElementsByTagName("Url").item(0));
				names.add(new String(text.getChildNodes().item(0).getNodeValue().getBytes("UTF-8"), "UTF-8"));
				info.add(URLDecoder.decode(urlnode.getChildNodes().item(0).getNodeValue(), "UTF-8"));
				//topics.add(new String(text.getChildNodes().item(0).getNodeValue().getBytes("UTF-8"), "UTF-8"));
			}
		}
		topics.add(names);
		topics.add(info);
		return topics;
	}
	
	/**
	 * PROTOTYPE - Inserts a desambiguation gadget
	 * @param blip The submitted blip
	 * @throws DOMException
	 * @throws SAXException
	 * @throws ParserConfigurationException
	 * @throws UnsupportedEncodingException
	 * @throws JSONException 
	 */
	private void insertGadget(Blip blip) throws DOMException, SAXException, ParserConfigurationException, UnsupportedEncodingException, JSONException{
		
		while(true){
		String text = blip.getDocument().getText();
		
		Matcher textmatcher = EXPERIMENTAL_EXPR.matcher(text);
		Matcher allmatcher = EXPERIMENTAL_ALL.matcher(text);
		
		
		if(textmatcher != null && allmatcher != null && textmatcher.find() && allmatcher.find()){
			
			ArrayList<ArrayList<String>> topics = null;
			String topic = new String(textmatcher.group(3).getBytes("UTF-8"), "UTF-8");
			String lang = textmatcher.group(2);
			if(lang == null)
				lang = "en";
			topic = topic.replaceAll(" ", "%20");
			URL searchurl;
			
			
			try {
				searchurl = new URL(String.format(WIKISEARCH, lang, topic));
				BufferedReader reader = new BufferedReader(new InputStreamReader(searchurl.openStream(), "UTF-8"));
				String data = "";
				String line = "";
				while((line = reader.readLine()) != null){
					data += line;
				}
				reader.close();
				
				topics = getAllTopics(data, textmatcher.group(3));

			} catch (MalformedURLException e) {
				log.severe(e.getMessage());
			} catch (IOException e) {
				log.severe(e.getMessage());
			}
			
			Gadget picker = new Gadget(GADGET_URL);
			if(topics != null){
				JSONArray array = new JSONArray(topics);
				log.severe(array.toString());
				String strarray = array.toString().replace('"', '\'');
				picker.setField("query", textmatcher.group(3));
				picker.setField("topics", strarray);
			}
			
			//blip.getDocument().insertElement(0, picker);
			blip.getDocument().delete(new Range(allmatcher.start(), allmatcher.end()));
			blip.getDocument().insertElement(allmatcher.start(), picker);
			/*log.warning(blip.getDocument().getGadgetView().getGadgets().size() + " gadgets");*/

		} else {
			return;
		}
		
		}
	}
	
	private void updateGadgets(Blip blip){
		List<Gadget> gadgets = blip.getDocument().getGadgetView().getGadgets();
		for(Gadget g : gadgets){
			if(g.getUrl().equalsIgnoreCase(GADGET_URL) && g.getField("selection") != null){
				String query = g.getField("query");
				String selection = g.getField("selection");
				int position = blip.getDocument().getPosition(g);
				AbstractFormatter formatter = new LinkFormatter();
				String text = formatter.format(query, selection);
				blip.getDocument().insert(position, text);
				blip.getDocument().getGadgetView().delete(g);
				log.severe("Deleted gadget " + g.toString());
			}
		}
	}
	
	/**
	 * Called when a blip is submitted
	 * @param blip The submitted blip
	 * @throws SAXException
	 * @throws ParserConfigurationException
	 * @throws UnsupportedEncodingException
	 * @throws JSONException 
	 * @throws DOMException 
	 */
	private void onBlipSubmitted(Blip blip) throws SAXException, ParserConfigurationException, UnsupportedEncodingException, DOMException, JSONException{

		replaceText(blip, Types.WIKIFY);
		replaceText(blip, Types.WIKIDEF);

		insertGadget(blip);
	}

	/**
	 * Main event processing function
	 */
	@Override
	public void processEvents(RobotMessageBundle bundle) {
		Wavelet wavelet = bundle.getWavelet();

		for (Event event : bundle.getEvents()){
			try {
				switch(event.getType()){
				case WAVELET_SELF_ADDED:
					Blip blip = wavelet.appendBlip();
					TextView textview = blip.getDocument();
					textview.append("Hi! I can add Wikipedia link for topics by typing <wikify topic>. " +
							"Additionally, I can insert full definitions using <wikidef topic>. ;)");
					break;
				//Must test real-time replacement better before using it for real.
				/*case DOCUMENT_CHANGED:
					updateGadgets(event.getBlip());
					break;*/
				case BLIP_SUBMITTED:
					if(!(event.getModifiedBy().equalsIgnoreCase("wikifier@appspot.com"))){
						updateGadgets(event.getBlip());
						onBlipSubmitted(event.getBlip());
					}
					break;
				default:
					break;
				}
				/* I should probably care about these exceptions somewhere :-) 
				 * I'll just log them for now. */
			} catch (SAXException e) {
				log.severe(e.getMessage());
			} catch (ParserConfigurationException e) {
				log.severe(e.getMessage());
			} catch (UnsupportedEncodingException e) {
				log.severe(e.getMessage());
			} catch (DOMException e) {
				log.severe(e.getMessage());
			} catch (JSONException e) {
				log.severe(e.getMessage());
			}
		}

	}

}
